# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html

import MySQLdb

from sqlalchemy.orm import sessionmaker
from sqlalchemy import create_engine
from zhihu_spider.models.detailModel import BaseZhihuUser, UserAboutUrls, DetailZhihuUser, Base


class ZhihuSpiderPipeline(object):
    def process_item(self, item, spider):
        return item


# 字段存储到Mysql
class MySQLStorePipeline(object):
    '''
    fix scrapy in linux
    error: command 'gcc' failed with exit status 1 的解决办法
    用pip安装软件时提示：error: command 'gcc' failed with exit status 1
    easy_install也提示一样的错误：error: Setup script exited with error: command 'gcc' failed with exit status 1
    一般是缺少python-dev包
    apt-get install python-dev或yum install python-devel这样就行了
    fix charset in simple chinese
    ALTER DATABASE databasename CHARACTER SET utf8 COLLATE utf8_unicode_ci;
    ALTER TABLE tablename CONVERT TO CHARACTER SET utf8 COLLATE utf8_unicode_ci;
    '''

    def process_item(self, item, spider):
        print "start insert data"

        engine = create_engine('mysql+mysqldb://root:toor123@115.29.148.86:3306/zhihu?charset=utf8', echo=True)
        Base.metadata.create_all(engine)  # 创建数据库表      这部分可以提取出去
        dbsession = sessionmaker(bind=engine)
        session = dbsession()
        zhi_id = item["zhi_id"]
        nick = item["nick"].encode('utf-8')
        location = item["location"].encode('utf-8')
        bussiness = item["bussiness"].encode('utf-8')
        sex = item["sex"]
        vote = int(item["vote"])
        thanks = int(item["thanks"])
        fav = int(item["fav"])
        share = int(item["share"])
        employment = item["employment"].encode('utf-8')
        position = item["position"].encode('utf-8')
        edu = item["edu"].encode('utf-8')
        edu_extra = item["edu_extra"].encode('utf-8')
        followees = int(item["followees"])
        followeers = int(item["followeers"])
        column = item["column"].encode('utf-8')
        topics = item["topics"].encode('utf-8')
        weibo = item["weibo"]
        topics_list = item["topics_list"]
        detail_info = DetailZhihuUser(zhi_id, nick, location, bussiness, sex, vote, thanks, fav, share,
                                      employment, position, edu, edu_extra, followees, followeers,
                                      column, topics, weibo, topics_list)
        try:
            session.add(detail_info)
            session.commit()
        except Exception:
            pass

        # session.close()
        return item


# 字段存储到Mongodb
class MongoDBPipline(object):
    def process_item(self, item, spider):
        """
        do nothing
        :param item:
        :param spider:
        :return:
        """
